import scrapy
from spidertools.utils.time_utils import get_current_date
import requests
from commonresources.spider_items.shandong.items import ShanDongShengShuiLIGongChengZhaoBiaoTouBiaoGongGongFuWuPingTaiItem
from commonresources.spiders.basespider import BaseSpider
from scrapy.http import Request
import time
import requests.packages.urllib3
from scrapy.selector import Selector
import re



class ShanDongShengShuiLiGongChengZhaoBiaoTouBiaoGongGongFuWuPingTaiSpider(BaseSpider):

    def __init__(self, full_dose=True):
        super(ShanDongShengShuiLiGongChengZhaoBiaoTouBiaoGongGongFuWuPingTaiSpider, self).__init__(full_dose)
    """
            山东省水利工程招标投标公共服务平台     http://wr.shandong.gov.cn/dzzb/
        """
    name = 'ShanDongShengShuiLiGongChengZhaoBiaoTouBiaoGongGongFuWuPingTai'
    name_zh = "山东省水利工程招标投标公共服务平台"
    province = "山东"

    def start_requests(self):

        start_urls = ['http://60.208.113.93:6889/slgcjypt/pub-PubAnn-list.do',#招标
            # 'http://60.208.113.93:6889/slgcjypt/pub-PubAnn-listSub.do',#变更
            # 'http://60.208.113.93:6889/slgcjypt/pub-PubAnn-listYc.do',#异常
            # 'http://60.208.113.93:6889/slgcjypt/pub-PubCand-list.do',#评标结果公示
                      #http://60.208.113.93:6889/slgcjypt/pub-PubAcc-list.do?accpubBean.tendname=&accpubBean.gsr=&pager.pageNo=1
            'http://60.208.113.93:6889/slgcjypt/pub-PubAcc-list.do'#中标结果
                  ]

        for type in start_urls:
            if type == 'http://60.208.113.93:6889/slgcjypt/pub-PubAnn-list.do':
                type_name = '招标公告'
            elif type == 'http://60.208.113.93:6889/slgcjypt/pub-PubAcc-list.do':
                type_name = '中标公告'
        page_number = 50
        yield scrapy.Request(
            url=type,
            method="get",
            meta={
                'first_url': type,
                "need_break": False,
                'type': type_name,
                'page': page_number
            },
            callback=self.first_parse,
        )

    def first_parse(self,response):
        second_page_number = 1
        headers = {
            'USER_AGENT': "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"
        }
        if not response.meta['need_break']:
            while second_page_number < int(response.meta['page']):
                if response.meta['type'] == '中标公告':
                    start_url = 'http://60.208.113.93:6889/slgcjypt/pub-PubAcc-list.do?accpubBean.tendname=&accpubBean.gsr=&pager.pageNo='+str(second_page_number)
                if response.meta['type']=='招标公告':
                    start_url='http://60.208.113.93:6889/slgcjypt/pub-PubAnn-list.do?noticeBean.pxmmc=&noticeBean.pdljg=&noticeBean.pubtimebegin=&noticeBean.pubtimeend=&pager.pageNo='+str(second_page_number)
                if not response.meta['need_break']:
                    html = requests.get(url=start_url, headers=headers, verify=False)
                    html.encoding = 'utf-8'
                    f = html.text
                    sel = Selector(text=f)
                    li_list = sel.xpath('//*[@id="inside_mian"]/div[@class="main_right"]/div[@class="news_list_i3"]/ul/li')
                    for li in li_list:
                        li_url = li.xpath('./a/@href')[0].extract()
                        li_title = li.xpath('./a/@title')[0].extract()
                        li_time = li.xpath('./span/text()')[0].extract()
                        li_time = self.time_change(li_time)
                        origin_url = 'http://60.208.113.93:6889' + li_url  # 详情页URL
                        requests.packages.urllib3.disable_warnings()  # 不显示报错
                        second_html = requests.get(url=origin_url, headers=headers, verify=False)#详情页的源码
                        second_html.encoding = 'utf-8'
                        html = second_html.text
                        item = ShanDongShengShuiLIGongChengZhaoBiaoTouBiaoGongGongFuWuPingTaiItem()
                        item['origin_url'] = origin_url  # 原始网站
                        item['announcement_title'] = li_title  # 公告标题
                        item['release_time'] = li_time  # 发布时间
                        item['announcement_type'] = response.meta['type']  # 公告类型
                        item['html'] = html  # 网页源码
                        item['source_type'] = '山东省水利工程招标投标公共服务平台' #属于哪个网站
                        item['province'] = '山东'
                        item['is_parsed'] = 0
                        # print(item)
                        time.sleep(1)
                        if not self.full_dose and item['release_time'] != get_current_date():
                            response.meta['need_break'] = True
                        else:
                            yield item
                if not response.meta['need_break']:
                    time.sleep(2)
                    second_page_number += 1
                else:
                    break


    def time_change(self,a):
        a = a.strip()
        a = a.replace('[', '')
        a = a.replace(']', '')
        a = a.strip()
        a = a.split(" ")[0]
        return a